#!/usr/local/python3/bin/python3
# -*- coding: UTF-8 -*-
'''
mandyctl.py
常驻内存运行, task_process 表中status in (1,9) 的作业启动和停止. 
daemon 进程实时监控其状态的变化.
'''
import pymysql
import sys
import time
import subprocess
import psutil
import ctypes
import os
import platform
import configparser
#  ------------------
from tools import *
from MmsLogging import get_logger

#取得最后的配置时间, 用于配置的版本控制, 有配置更新的话, 就要重新加载配置.
#last_cfg_time 通过cfg_job的触发器更新, 其实只影响daemon作业
def get_last_cfg_time(mydb):
    str_sql = "select modify_time from monitor_global where param_name = 'last_cfg_time' limit 1"
    #print("str_sql:", str_sql, " line:", sys._getframe().f_lineno)
    mycursor = mydb.cursor()
    try:
       count = mycursor.execute(str_sql)
       database_data = mycursor.fetchone()
    except:
       mandy_logger.critical("mycursor.execute(str_sql):%s line:%d"% (str_sql, sys._getframe().f_lineno))
       quit()
    finally:
       mycursor.close()
    
    if count == 1:
        return database_data[0]
    
    return 'default'
    
    
#insert_host_metrics(mydb, host_name, cpu_percent, mem_percent, disk_percent)
def insert_host_metrics(mydb, host_name, cpu_percent, mem_percent, disk_percent):
    '''
       插入一条新的主机度量指标数据。
    '''
    mycursor = mydb.cursor()
    # SQL 插入语句
    str_sql = "INSERT INTO host_metrics(host_name, cpu_percent, mem_percent, disk_percent) \
           VALUES ('%s', '%d', %d, '%d')" % (host_name, cpu_percent, mem_percent, disk_percent)
    try:
       # 执行sql语句
       mycursor.execute(str_sql)
       mandy_logger.debug("mycursor.execute(str_sql):%s line:%d"% (str_sql, sys._getframe().f_lineno))
       # 执行sql语句
       mydb.commit()
    except:
       # 发生错误时回滚
       mydb.rollback()
       mandy_logger.critical("%s line:%d"% (str_sql, sys._getframe().f_lineno))
       quit()
    finally:
       mycursor.close()

def get_cfg_host_info(mydb, host_name):
    '''
        取得主机配置数据, 主要是 max_concurrent 
    '''
    count = 0
    mycursor = mydb.cursor()
    try:
        # name 是主键, 只返回一条数据
        str_sql = "select name, ip_addr, max_concurrent from monitor_host where name ='%s' and iz_enabled = '1' limit 1" %(host_name)
        # str_sql = "select name, ip_addr from cfg_host where name ='" +host_name + "' "
        mandy_logger.debug("str_sql: %s" %(str_sql))
        count = mycursor.execute(str_sql)
        cfg_host_data = mycursor.fetchone()
    except:
        mandy_logger.critical("unable to fetch data: ", sys._getframe().f_lineno)
        quit()
    finally:
        mycursor.close()
        
    # mydb.close()
    if count != 1 :
        mandy_logger.critical("no host's data, pls check!")
        quit()
    
    cfg_host = {}
    cfg_host['name'] = cfg_host_data[0]
    cfg_host['ip_addr'] = cfg_host_data[1]
    cfg_host['max_concurrent'] = cfg_host_data[2]
    return cfg_host

def get_daemon_running_pid(mydb, daemon_jobs):
    '''
    从主机中获得所有运行的进程
    进行比较, 确定进程状态  running_pid
    '''
    # mms_logger = get_logger()
    # 取得当前正常运行的pids, 逐个 daemon_jobs 去匹配
    for id in daemon_jobs.keys():
        daemon_jobs[id]['running_pid'] = -1
        
    psutil_pids = psutil.pids()
    match_cfg_job_cnt = 0
    for each_pid in psutil_pids:
        try:
            p = psutil.Process(each_pid)
            ps_cmdline = p.cmdline()  # 返回的也是个列表, 命令行的每个字, 
            for id in daemon_jobs.keys():
                if (daemon_jobs[id]['running_pid'] == -1):
                    #'ps_key_words': 'python3,test_timer.py,test_daemon_job_2',
                    job_cfg_keys = daemon_jobs[id]['ps_key_words']
                    match_cfg_keyworld_cnt = 0
                    for cfg_keyworld in job_cfg_keys:
                        for ps_keyworld in ps_cmdline:
                            if cfg_keyworld.strip() in ps_keyworld:
                                match_cfg_keyworld_cnt += 1
                                break;
                    
                    if (match_cfg_keyworld_cnt == len(job_cfg_keys) ):
                        daemon_jobs[id]['running_pid'] = each_pid
                        match_cfg_job_cnt += 1
        except:
            continue
        
        # 全部配置的进程都找到匹配了，不用再循环
        if match_cfg_job_cnt == len(daemon_jobs):
            break;
            
def get_launch_normal_tasks(mydb, host_name, max_task):
    '''
       取回launch状态的任务,  max_task 用来限定最大条目
    '''
    str_limit = ""
    if max_task != 0:
        str_limit = " limit %d" %(max_task)
    
    count = 0
    mycursor = mydb.cursor()
    fetch_dataset = []
    try:
        # monitor_job.iz_canceled 状态只影响生成 monitor_task 任务, 不影响单次任务的执行判断. 
        str_sql = "select a.id as task_id, a.job_id as job_id, a.start_command  \
                   ,a.parent_id as parent_task_id, a.old_id as old_task_id \
                  ,trim(ifnull(a.log_file_name,'')) as log_file_name, DATE_FORMAT(a.modify_time, '%%Y-%%m-%%d %%T') as modify_time \
                  from monitor_task a\
                  where a.status = '1' and a.host_name = '%s' and a.action_type in ('0', '2') \
                      and a.job_type = '3' order by a.id %s " %(host_name, str_limit)
        mandy_logger.debug("mycursor.execute(str_sql): %s" %(str_sql))
        count = mycursor.execute(str_sql)
        fetch_dataset = mycursor.fetchall()
    except:
        mandy_logger.critical("str_sql: %s" %(str_sql))
        mandy_logger.critical("unable to fetch data: %d" % (sys._getframe().f_lineno))
        quit()
    finally:
        mycursor.close()
        
    # mydb.close()
    tasks = {}
    for data in fetch_dataset:
        task = {}
        task['task_id'] = data[0]
        task['job_id'] = data[1]
        task['start_command'] = data[2]
        task['parent_task_id'] = data[3]
        task['old_task_id'] = data[4]
        task['log_file_name'] = data[5]
        task['modify_time'] = data[6]
        tasks[task['task_id']] = task
    return tasks

def get_shutdown_normal_tasks(mydb, host_name):
    '''
       取回shutdown状态的任务(status in '9','w'), max_task 用来限定最大条目
    '''
    count = 0
    mycursor = mydb.cursor()
    fetch_dataset = []
    try:
        # monitor_job.iz_canceled 状态只影响生成 monitor_task 任务, 不影响单次任务的执行判断.
        str_sql = "select a.id as task_id, a.job_id as job_id, case when ifnull(a.stop_command, '') = '' then 'KILL_PID' else a.stop_command end as stop_command \
                          ,a.status as status, ifnull(a.instance_pid, -1) as instance_pid, DATE_FORMAT(a.modify_time, '%%Y-%%m-%%d %%T') as modify_time \
                   from monitor_task a \
                   where a.`status` in ('9', 'w') and a.host_name = '%s'  \
                         and a.job_type = '3' order by a.id" %(host_name)
        mandy_logger.debug("mycursor.execute(str_sql): %s" %(str_sql))
        count = mycursor.execute(str_sql)
        fetch_dataset = mycursor.fetchall()
    except:
        mandy_logger.error("str_sql: %s" %(str_sql))
        mandy_logger.error("unable to fetch data: %d" % sys._getframe().f_lineno)
    finally:
        mycursor.close()
        
    # mydb.close()
    tasks = {}
    for data in fetch_dataset:
        task = {}
        task['job_type'] = '3'
        task['task_id'] = data[0]
        task['job_id'] = data[1]
        task['stop_command'] = data[2]
        task['status'] = data[3]
        task['instance_pid'] = data[4]
        task['modify_time'] = data[5]
        tasks[task['task_id']] = task
    return tasks
    
    
def start_normal_task(running_tasks, log_dir, this_task):
    global mandy_logger
    task_id = this_task['task_id']
    parent_task_id = this_task['parent_task_id']
    job_id = this_task['job_id']
    str_now = time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime())
    if (this_task['log_file_name'] == ''):
        log_file_name = 'job_%d_%d_%d_%s.log' % (job_id, parent_task_id, task_id, str_now)
        full_log_file_name = os.path.join(log_dir, log_file_name)
        log_file = open(full_log_file_name, "a+")
    else:#有自己设定的日志文件名, 不要标准输出的作日志。
        log_file_name = 'none'
        log_file = -1
    
    # log_file_name = os.path.join(log_dir, log_file_name)  改了, 不合并, 安全考虑读日志只能读配置目录上的, #合并路径名字和文件名字
    command = this_task['start_command']
    mandy_logger.info("TASK START:task_id:%d, job_id:%d, start_command:%s" % (task_id, job_id, command))
    # log_file = open(log_file_name, "ab+")
    # sub_p = subprocess.Popen(command, shell = True, stdout = log_file, stderr = subprocess.STDOUT)
    # print("command.split():", command.split(" "))
    if (log_file != -1):
        sub_p = subprocess.Popen(command.split(" "), stdout = log_file, stderr = log_file, bufsize=1)
    else:
        sub_p = subprocess.Popen(command.split(" "))
        
    # 往数据库task_process表里更新任务状态. 状态由 1 变为 2
    task_status = {}
    task_status['task_id'] = task_id
    task_status['status'] = '2'
    #task_status['start_command'] = command
    if (log_file_name != 'none'):
        task_status['log_file_name'] = log_file_name
    task_status['start_time'] = 'now'
    task_status['instance_pid'] = sub_p.pid
    task_status['modify_time'] = this_task['modify_time']
    ret = update_task_process(mydb, mandy_logger, task_status)
    if (ret != 0):
        mandy_logger.error("update status failed after task start. task_id:%d, line: %d" % (task_id, sys._getframe().f_lineno))
        shutdown_normal_task(mydb, task_status)
        
    # 往字典里记录本子进程信息, 需要 poll(), wait(), 关闭它
    job_task_dict = {}
    job_task_dict['log_file_handle'] = log_file
    job_task_dict['log_file_name'] = log_file_name
    job_task_dict['start_command'] = command
    job_task_dict['p_subprocess'] = sub_p
    job_task_dict['job_id'] = job_id
    job_task_dict['parent_task_id'] = parent_task_id
    job_task_dict['status'] = '2'
    job_task_dict['instance_pid'] = sub_p.pid
    job_task_dict['task_id'] = task_id
    running_tasks[job_task_dict['instance_pid']] = job_task_dict
    
def shutdown_normal_task(mydb, this_task):
    command = 'echo hello'
    if (this_task['stop_command'] == 'KILL_PID'):
        command = 'kill %d &' %(this_task['instance_pid'])
    else:
        command = "%s > /dev/null 2>&1 &" % (this_task['stop_command'])
        
    mandy_logger.info("TASK STOP, task_id:%d job_id:%d status:%s command:%s " % (this_task['task_id'], this_task['job_id'], this_task['status'], command))
    sub_p = subprocess.Popen(command, shell = True)
    task_process = {}
    task_process['task_id'] = this_task['task_id']
    task_process['modify_time'] = this_task['modify_time']
    task_process['status'] = 'w'
    task_process['stop_time'] = 'now'
    update_task_process(mydb, mandy_logger, task_process)
    
def normal_tasks_poll(running_tasks):
    global mandy_logger
    quit_instance_pids = []
    for instance_pid in running_tasks.keys():
        job_task_dict = running_tasks[instance_pid]
        p_subprocess = job_task_dict['p_subprocess']
        if p_subprocess.poll() is not None:
            #有一个结束了,退出 for
            #TODO 修改为实时写，而不是退出写
            p_subprocess.wait()
            mandy_logger.debug("p_subprocess.wait(),job_task_dict['log_file_name']: %s" % (job_task_dict['log_file_name']))
            fo = job_task_dict['log_file_handle']
            if (fo != -1):
                #popen = job_task_dict['p_subprocess']
                #r = popen.stdout.readline().decode('gbk')
                lastLine = "MANDY_CTL_LOG_END.\n"
                fo.write(lastLine)
                fo.close()
                
            task_status = {}
            task_status['task_id'] = job_task_dict['task_id']
            task_status['status'] = '3'
            #job_task_dict['stop_time'] 会被替换成 now()
            task_status['stop_time'] = 'now'
            task_status['return_code'] = p_subprocess.returncode
            mandy_logger.info("TASK FINISH: task_id:%d, job_id:%d, start_command:%s"
                            % (job_task_dict['task_id'], job_task_dict['job_id'],  job_task_dict['start_command']))
            #print("job_task_dict:", job_task_dict)
            update_task_process(mydb, mandy_logger, task_status)
            quit_instance_pids.append(instance_pid)
        #else:
        #    fo = job_task_dict['log_file_handle']
        #    if (fo != -1):
        #        popen = job_task_dict['p_subprocess']
        #        r = popen.stdout.readline().decode('gbk')
        #        fo.write(r)
        #        fo.flush()
            #sys.stdout.write(r)
    
    for quit_instance_pid in quit_instance_pids:
        running_tasks.pop(quit_instance_pid)
        


def get_daemon_job_status(mydb, jobId):
    mycursor = mydb.cursor()
    str_sql = ""
    job_status_record = []
    try:# cfg_job.iz_canceled 状态只影响生成 task_process 任务, 不影响单次任务的执行判断.
        str_sql = "select ifnull(t.id, -1) as task_id, t.action_type as action_type, t.status as status, t.instance_pid as instance_pid, \
             t.conflict_flag as conflict_flag, ifnull(c.realtime_instance_pid, -1) as realtime_instance_pid ,\
             DATE_FORMAT(t.modify_time, '%%Y-%%m-%%d %%T') as modify_time, t.start_command as start_command, \
             case when ifnull(t.stop_command,'') = '' then 'KILL_PID' else t.stop_command end as stop_command, \
             c.iz_enabled as iz_enabled \
            from monitor_job c  \
            left join monitor_task t on c.current_task_id = t.id where c.id = %d " %(jobId)
            
        count = mycursor.execute(str_sql)
        job_status_record = mycursor.fetchone()
    except:
        mandy_logger.error("mycursor.execute(str_sql) hhh: %s  line:%d" %(str_sql, sys._getframe().f_lineno))
    finally:
        mycursor.close()
        
    print("job_status_record:", job_status_record)
    jobStatus = {}
    if (len(job_status_record) > 0):
        jobStatus['task_id'] = job_status_record[0]
        jobStatus['action_type'] = job_status_record[1]
        jobStatus['old_status'] = job_status_record[2]
        jobStatus['instance_pid'] = job_status_record[3]
        jobStatus['conflict_flag'] = job_status_record[4]
        jobStatus['realtime_instance_pid'] = job_status_record[5]
        jobStatus['modify_time'] = job_status_record[6]
        jobStatus['start_command'] = job_status_record[7]
        jobStatus['stop_command'] = job_status_record[8]
        jobStatus['iz_enabled'] = job_status_record[9]
        jobStatus['job_id'] = jobId
    
    return jobStatus
    
    
def handle_daemon_task(mydb, daemon_jobs):
#def update_daemon_task(mydb, daemon_jobs):
    for each_daemon in daemon_jobs.values():
        jobStatus = get_daemon_job_status(mydb, each_daemon['job_id'])
        if (not 'task_id' in jobStatus):
            continue
        
        if (jobStatus['task_id'] == -1 or jobStatus['iz_enabled'] == '0'):
            # 是新配置的作业或者是不再启用的作业
            continue
        
        if (jobStatus['realtime_instance_pid'] == each_daemon['running_pid'] and jobStatus['old_status'] == each_daemon['old_status']):
            # 与上一次的是一样的, 不需要再处理
            continue
        
        if (jobStatus['conflict_flag'] == '1'):
            #当前任务已经有了冲突
            if (jobStatus['realtime_instance_pid'] == each_daemon['running_pid']):
                #此次的进程号与上一次的是一样的, 无需处理
                continue
            else:
                #此次的进程号与上一次的不一样, 只要记录一下新的进程号到remark里边, 同时更新 job_status 表
                task_process_dict = {}
                task_process_dict['task_id'] = jobStatus['task_id']
                task_process_dict['modify_time'] = jobStatus['modify_time']
                task_process_dict['remark'] = "new_pid:%d" %(each_daemon['running_pid'])
                ret = update_task_process(mydb, mandy_logger, task_process_dict)
                if (ret == 0):
                    job_status_new = {}
                    job_status_new['job_id'] = jobStatus['job_id']
                    job_status_new['realtime_instance_pid'] = each_daemon['running_pid']
                    update_realtime_instance_pid(mydb, mandy_logger, job_status_new)
                else:
                    mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
                    
            each_daemon['old_status'] = jobStatus['old_status']
            continue
        
        # 0 未运行, 1 就绪状态执行运行命令, 2 正在运行, 3 运行正常结束, 4,  错过停止时间  5 运行异常结束, 6 错过运行时间, 7 等待启动条件成立, 8 作业超时, 9 等待执行停止命令,
        # c 已经执行运行命令, 待检测运行状态.  w 停止命令已经执行, 等待进程结束. e 未运行就停止, d 无法停止, w状态如果持续超过1分钟
        if (jobStatus['old_status'] == 'c'):
            # c 已经执行运行命令, 待检测运行状态
            if (each_daemon['running_pid'] == -1):
                #还没启动成功
                mandy_logger.info("checking task_id: %d" % (jobStatus['task_id']))
            else:
                #启动成功
                task_process_dict = {}
                task_process_dict['task_id'] = jobStatus['task_id']
                task_process_dict['modify_time'] = jobStatus['modify_time']
                task_process_dict['status'] = '2'
                task_process_dict['instance_pid'] = each_daemon['running_pid']
                mandy_logger.info("start success task_id: %d, instance_pid:%d" % (jobStatus['task_id'], each_daemon['running_pid']))
                ret = update_task_process(mydb, mandy_logger, task_process_dict)
                if (ret == 0):
                    each_daemon['old_status'] = '2'
                    job_status_new = {}
                    job_status_new['job_id'] = jobStatus['job_id']
                    job_status_new['realtime_instance_pid'] = each_daemon['running_pid']
                    update_realtime_instance_pid(mydb, mandy_logger, job_status_new)
                else:
                    mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
            continue
        elif (jobStatus['old_status'] == '0'):
            # 0 未运行, 此状态不会促使作业变更当前任务记录.
            mandy_logger.error("status = 0, unexpected job status. task_id:%d" %(jobStatus['task_id']))
            continue
        elif (jobStatus['old_status'] == '1'):
            # 运行启动命令
            # 启动后状态是 'c'
            if (each_daemon['running_pid'] == -1):
                # 启动 new_status = 'c'
                command = "%s > /dev/null 2>&1 &" % (jobStatus['start_command'])
                mandy_logger.info("DAEMON TASK START, task_id:%d job_id:%d command:%s " % (jobStatus['task_id'], jobStatus['job_id'], command))
                sub_p = subprocess.Popen(command, shell = True)
                task_process_dict = {}
                task_process_dict['task_id'] = jobStatus['task_id']
                task_process_dict['modify_time'] = jobStatus['modify_time']
                task_process_dict['status'] = 'c'
                task_process_dict['start_time'] = 'now'
                ret = update_task_process(mydb, mandy_logger, task_process_dict)
                if (ret == 0):
                    each_daemon['old_status'] = 'c'
                    job_status_new = {}
                    job_status_new['job_id'] = jobStatus['job_id']
                    job_status_new['realtime_instance_pid'] = each_daemon['running_pid']
                    update_realtime_instance_pid(mydb, mandy_logger, job_status_new)
                else:
                    mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
            else:
                #已经在运行，无需再启动, remark 中作备注, 但不能标conflict_flag
                task_process_dict = {}
                task_process_dict['task_id'] = jobStatus['task_id']
                task_process_dict['modify_time'] = jobStatus['modify_time']
                task_process_dict['status'] = '2'
                task_process_dict['start_time'] = 'now'
                task_process_dict['instance_pid'] = each_daemon['running_pid']
                task_process_dict['remark'] = "already exist pid:%d" %(each_daemon['running_pid'])
                ret = update_task_process(mydb, mandy_logger, task_process_dict)
                if (ret == 0):
                    each_daemon['old_status'] = '2'
                    job_status_new = {}
                    job_status_new['job_id'] = jobStatus['job_id']
                    job_status_new['realtime_instance_pid'] = each_daemon['running_pid']
                    update_realtime_instance_pid(mydb, mandy_logger, job_status_new)
                else:
                    mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
            continue
        elif (jobStatus['old_status'] == '2' or jobStatus['old_status'] == '8'):
            if (each_daemon['running_pid'] == -1 or jobStatus['realtime_instance_pid'] != each_daemon['running_pid']):
                # 原先在运行, 现在停止了或者进程号变了, 状态设为5, 异常停止
                # 后台会判断并插入一条新的状态为 0 未运行的重启记录, 待时间到了会重启
                task_process_dict = {}
                task_process_dict['task_id'] = jobStatus['task_id']
                task_process_dict['modify_time'] = jobStatus['modify_time']
                task_process_dict['conflict_flag'] = '1'
                task_process_dict['stop_time'] = 'now'
                task_process_dict['status'] = '5'
                task_process_dict['remark'] = "new_pid:%d" %(each_daemon['running_pid'])
                ret = update_task_process(mydb, mandy_logger, task_process_dict)
                if (ret == 0):
                    each_daemon['old_status'] = '5'
                    job_status_new = {}
                    job_status_new['job_id'] = jobStatus['job_id']
                    job_status_new['realtime_instance_pid'] = each_daemon['running_pid']
                    update_realtime_instance_pid(mydb, mandy_logger, job_status_new)
                else:
                    mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
            continue
        elif (jobStatus['old_status'] == '3' or jobStatus['old_status'] == '5' or jobStatus['old_status'] == 'e'):
            # 3 运行正常结束, 5 异常停止  e 未运行就取消
            if (jobStatus['realtime_instance_pid'] != each_daemon['running_pid']):
                task_process_dict = {}
                task_process_dict['task_id'] = jobStatus['task_id']
                task_process_dict['conflict_flag'] = '1'
                task_process_dict['remark'] = "new_pid:%d" %(each_daemon['running_pid'])
                ret = update_task_process(mydb, mandy_logger, task_process_dict)
                if (ret == 0):
                    each_daemon['old_status'] = '3'
                    job_status_new = {}
                    job_status_new['job_id'] = jobStatus['job_id']
                    job_status_new['realtime_instance_pid'] = each_daemon['running_pid']
                    update_realtime_instance_pid(mydb, mandy_logger, job_status_new)
                else:
                    mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
            continue
        elif (jobStatus['old_status'] == '7'):
            # 7 等待启动条件成立
            continue
        elif (jobStatus['old_status'] == '9'):
            # 9 等待执行停止命令
            if (each_daemon['running_pid'] == -1 or jobStatus['realtime_instance_pid'] != each_daemon['running_pid']):
                # 已经处于停止状态
                task_process_dict = {}
                task_process_dict['task_id'] = jobStatus['task_id']
                task_process_dict['modify_time'] = jobStatus['modify_time']
                task_process_dict['conflict_flag'] = '1'
                task_process_dict['stop_time'] = 'now'
                task_process_dict['status'] = '5'
                task_process_dict['remark'] = "new_pid:%d" %(each_daemon['running_pid'])
                ret = update_task_process(mydb, mandy_logger, task_process_dict)
                if (ret == 0):
                    each_daemon['old_status'] = '5'
                    job_status_new = {}
                    job_status_new['job_id'] = jobStatus['job_id']
                    job_status_new['realtime_instance_pid'] = each_daemon['running_pid']
                    update_realtime_instance_pid(mydb, mandy_logger, job_status_new)
                else:
                    mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
            else:
                if (jobStatus['stop_command'] == 'KILL_PID'):
                   command = 'kill %d &' %(each_daemon['running_pid'])
                else:
                   command = "%s > /dev/null 2>&1 &" % (jobStatus['stop_command'])
                
                sub_p = subprocess.Popen(command, shell = True)
                task_process_dict = {}
                task_process_dict['task_id'] = jobStatus['task_id']
                task_process_dict['modify_time'] = jobStatus['modify_time']
                task_process_dict['stop_time'] = 'now'
                task_process_dict['status'] = 'w'
                ret = update_task_process(mydb, mandy_logger, task_process_dict)
                if (ret == 0):
                    each_daemon['old_status'] = 'w'
                else:
                    mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
                continue
        elif (jobStatus['old_status'] == 'w' or jobStatus['old_status'] == 'd'):
            # w 停止命令已经执行, 等待进程结束
            if (each_daemon['running_pid'] == -1 or jobStatus['realtime_instance_pid'] != each_daemon['running_pid']):
                # 已经处于停止状态
                job_status_new = {}
                job_status_new['job_id'] = jobStatus['job_id']
                job_status_new['realtime_instance_pid'] = each_daemon['running_pid']
                update_realtime_instance_pid(mydb, mandy_logger, job_status_new)
                    
                    
                task_process_dict = {}
                task_process_dict['task_id'] = jobStatus['task_id']
                task_process_dict['modify_time'] = jobStatus['modify_time']
                task_process_dict['stop_time'] = 'now'
                task_process_dict['status'] = '3'
                if (each_daemon['running_pid'] != -1):
                    task_process_dict['conflict_flag'] = '1'
                    task_process_dict['remark'] = "new_pid:%d" %(each_daemon['running_pid'])
                    
                ret = update_task_process(mydb, mandy_logger, task_process_dict)
                if (ret == 0):
                    each_daemon['old_status'] = '3'
                else:
                    mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
            else:
                if (jobStatus['old_status'] == 'w'):
                    #w需要再运行停止命令, d 就不需要了
                    if (jobStatus['stop_command'] == 'KILL_PID'):
                       command = 'kill %d &' %(each_daemon['running_pid'])
                    else:
                       command = "%s > /dev/null 2>&1 &" % (jobStatus['stop_command'])
                    sub_p = subprocess.Popen(command, shell = True)
                    task_process_dict = {}
                    task_process_dict['task_id'] = jobStatus['task_id']
                    task_process_dict['modify_time'] = jobStatus['modify_time']
                    task_process_dict['stop_time'] = 'now'
                    task_process_dict['stop_command'] = command
                    task_process_dict['status'] = 'w'
                    ret = update_task_process(mydb, mandy_logger, task_process_dict)
                    if (ret == 0):
                        each_daemon['old_status'] = 'w'
                    else:
                        mandy_logger.error("update_task_process, ret:%d, line:%d" %(ret, sys._getframe().f_lineno))
            
            continue
            
            
            
def get_cfg_job_daemon(mydb, host_name):
    count = 0
    mycursor = mydb.cursor()
    cfg_jobs = []
    str_sql = ""
    try:
        str_sql = "select job_name, case when (ifnull(ps_key_words, '') = '') then 'no_process_status_key_word' else ps_key_words end as ps_key_words, "\
                  " id as job_id, start_command , stop_command from monitor_job where iz_enabled = '1' and job_type = '1' and host_name ='%s'" % (host_name)
        count = mycursor.execute(str_sql)
        cfg_jobs = mycursor.fetchall()
    except :
        mandy_logger.critical("Error:%s line:%d"% (str_sql, sys._getframe().f_lineno))
        raise
        quit()
    finally:
        mycursor.close()
    
    daemon_jobs = {}
    for cfg_job in cfg_jobs:
        daemon_job = {}
        daemon_job['job_name'] = cfg_job[0]
        daemon_job['ps_key_words'] = cfg_job[1].split(',')
        daemon_job['job_id'] = cfg_job[2]
        daemon_job['start_command'] = cfg_job[3]
        daemon_job['stop_command'] = cfg_job[4]
        daemon_job['host_name'] = host_name
        daemon_job['old_status'] = '?'
        daemon_job['last_running_pid'] = -2 
        daemon_jobs[daemon_job['job_id']] = daemon_job
        
    return daemon_jobs
    
# ************************************************************************
# main()
if __name__ == "__main__":
    log_file_max_line = 50000
    
    str_now = time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime())
    log_file_name = 'mandyctl_%s.log' % (str_now)
    global_cfg = get_ini_section_items(os.path.abspath('./config') + "/mandyctl.ini", "mandyctl")
    mandy_logger=get_logger(global_cfg['log_dir'], global_cfg['log_level'], log_file_name)
    mandy_logger.info("MANDY_CTL START!")
    mandy_logger.debug("global_cfg:%s"% str(global_cfg))
    ret = get_global_db_connect(global_cfg, mandy_logger)
    if ret[0] != 0:
        mandy_logger.info("MANDY_CTL STOP! Database connect failed!")
        quit()
    mydb = ret[1]
    
    old_last_cfg_time = get_last_cfg_time(mydb)
    cfg_host = get_cfg_host_info(mydb, global_cfg['host_name'])
    global_dict = get_global_param(mydb, mandy_logger)
    daemon_jobs = get_cfg_job_daemon(mydb, cfg_host['name'])
    
    running_tasks = {}
    while True:
        begin_time = int(time.time())
        mandy_logger.debug("Main loop start : %s"%(time.ctime()))
        
        # 获取该主机所有的后台进程和下一个需要启动的进程
        # 为减少每次重复获取数据, 先取得最近更新时间，如果同上一次一样
        # 则配置没有更改;
        new_last_cfg_time = get_last_cfg_time(mydb)
        if (old_last_cfg_time != new_last_cfg_time):
           old_last_cfg_time = new_last_cfg_time;
           cfg_host = get_cfg_host_info(mydb, global_cfg['host_name'])
           global_dict = get_global_param(mydb, mandy_logger)
           daemon_jobs = get_cfg_job_daemon(mydb, cfg_host['name'])
           
        get_daemon_running_pid(mydb, daemon_jobs)
        handle_daemon_task(mydb, daemon_jobs)
        
        #检测有没有进程已经结束
        normal_tasks_poll(running_tasks)
        
        #从 task 中获得需要运行的普通job.
        max_task = cfg_host['max_concurrent'] - len(running_tasks)
        if (max_task > 0):
            normal_tasks = get_launch_normal_tasks(mydb, global_cfg['host_name'], max_task)
            for each_task in normal_tasks.values():
                start_normal_task(running_tasks, global_cfg['log_dir'], each_task)
            
        shutdown_normal_tasks = get_shutdown_normal_tasks(mydb, global_cfg['host_name'])
        for each_task in shutdown_normal_tasks.values():
            shutdown_normal_task(mydb, each_task)
            
        #mandy_logger.debug("daemon_jobs:%s, line:%d"%(str(daemon_jobs), sys._getframe().f_lineno))
        end_time = int(time.time())
        val = int(global_dict['metrics_interval']) + begin_time - end_time
        if val <=0:
            val = 1
        time.sleep(val)
        #cpu_percent = int(psutil.cpu_percent(interval = val, percpu=False))
        #
        #mem_percent = int(psutil.virtual_memory().percent)
        #disk_percent = int(psutil.disk_usage('/').percent)
        #insert_host_metrics(mydb, cfg_host['name'], cpu_percent, mem_percent, disk_percent)
